package com.cheng.example;

import com.cheng.spider.component.AbstractSpider;
import com.cheng.spider.component.SimpleSpider;
import com.cheng.spider.custom.OfferableUrlSource;
import com.cheng.spider.custom.SingleUrlSource;
import com.cheng.spider.custom.UrlSource;
import com.cheng.spider.extract.DataResolver;
import com.cheng.spider.extract.novel.NovelExtractor;
import com.cheng.spider.extract.url.UrlExtractor;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.junit.Test;

import java.util.LinkedList;
import java.util.List;

public class ExampleSpider {
    @Test
    public void test() {
        OfferableUrlSource urlSource = new OfferableUrlSource("https://www.biduo.cc/");

        AbstractSpider firstPageSpider = new SimpleSpider() {
            @Override
            protected UrlSource initUrlSource() {
                return new SingleUrlSource("https://www.biduo.cc/biquge/53_53723/");
            }

            @Override
            protected DataResolver initDataResolver() {
                return new UrlExtractor() {
                    @Override
                    protected List<String> extractUrlList(Document document) {
                        List<String> list = new LinkedList<>();
                        Elements elements = document.select("div#list > dl > dd > a");
                        elements.forEach(e -> {
                            list.add(e.attr("href"));
                        });
                        return list;
                    }

                    @Override
                    protected void resolveUrl(String url) {
                        urlSource.offer(url);
                    }
                };
            }

        };

        firstPageSpider.run();

        AbstractSpider contentSpider = new SimpleSpider() {
            @Override
            protected UrlSource initUrlSource() {
                return urlSource;
            }

            @Override
            protected DataResolver initDataResolver() {
                return new NovelExtractor() {
                    @Override
                    protected String extractContent(Document document) {
                        String[] ss = document
                                .select("div#content")
                                .html()
                                .replace("&nbsp;&nbsp;&nbsp;&nbsp;", "")
                                .split("<br>");
                        System.out.println(ss.length);
                        StringBuilder stringBuilder = new StringBuilder(4096);
                        for (int i = 0; i < ss.length; i++) {
                            if (i != 0)
                                stringBuilder.append("    ");
                            stringBuilder.append(ss[i]);
                        }
                        return stringBuilder.append("\n\n").toString();
                    }

                    @Override
                    protected String extractTitle(Document document) {
                        return document.title();
                    }

                    @Override
                    protected String getNovelFileName() {
                        return "测试文件.txt";
                    }
                };
            }
        };

        contentSpider.run();
    }
}
